{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "d7429b93",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.experimental import WideDeepModel,LinearModel\n",
    "from keras.models import Sequential\n",
    "import numpy as np\n",
    "import levenberg_marquardt as lm\n",
    "import pandas as pd\n",
    "from itertools import product\n",
    "from keras.optimizers import Adam,SGD,Adagrad\n",
    "from keras.layers import Dense,Conv1D,Conv2D,Flatten,MaxPool1D,LeakyReLU,MaxPooling1D,BatchNormalization\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import MinMaxScaler,StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "01408bbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def import_dataset(normalised=True,scaleMethod='Standard'):\n",
    "    '''\n",
    "    Imports Dataset and returns either scaled values depending upon user inputs\n",
    "    \n",
    "    Input:\n",
    "        normalised -- boolean depending upon whether the user wants to scale the values\n",
    "        scaleMethod -- Type of scaler to be used if normalised is True\n",
    "    \n",
    "    Output:\n",
    "        (X_train,X_test,Y_train,Y_test) -- the training and testing dataset\n",
    "        scaler -- used to perform inverse transform if dataset is scaled\n",
    "    '''\n",
    "    data = pd.read_csv('Dataset/Static_Model/15000DwithQuat.csv')\n",
    "    dataS = data.drop('Unnamed: 0',axis=1)\n",
    "    \n",
    "    if normalised == False:\n",
    "        scaler = 'None'\n",
    "        X = dataS.iloc[:,:7].values\n",
    "        Y = dataS.iloc[:,7:].values\n",
    "        X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size=0.15,random_state=0)\n",
    "        \n",
    "    elif scaleMethod == 'Standard':\n",
    "        scaler = StandardScaler()\n",
    "        scaler.fit(dataS)\n",
    "        dataS = scaler.transform(dataS)\n",
    "        X = dataS[:,:7]\n",
    "        Y = dataS[:,7:]\n",
    "        X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size=0.15,random_state=0)\n",
    "    \n",
    "    elif scaleMethod == 'MinMax':\n",
    "        scaler = MinMaxScaler(feature_range=(0,1))\n",
    "        scaler.fit(dataS)\n",
    "        dataS = scaler.transform(dataS)\n",
    "        X = dataS[:,:7]\n",
    "        Y = dataS[:,7:]\n",
    "        X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size=0.15,random_state=0)\n",
    "    \n",
    "    return X_train,X_test,Y_train,Y_test,scaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a48d959a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def inverseTransform(scaler,*arr):\n",
    "    '''\n",
    "    Used to perform Inverse Transformation on normalised dataset\n",
    "    \n",
    "    Input:\n",
    "        scaler -- Instance of Normaliser used\n",
    "        *arr -- list of arrays to be concatenated\n",
    "    '''\n",
    "    data = np.concatenate(arr,axis=1)\n",
    "    data = pd.DataFrame(data)\n",
    "    arrInverse = scaler.inverse_transform(data)\n",
    "    \n",
    "    return arrInverse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0cb60d5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost(y_test,y_pred):\n",
    "    '''\n",
    "    Calculates error of the model\n",
    "    '''\n",
    "    error = (y_test-y_pred)/y_test\n",
    "    error = np.sum(abs(error))/(y_test.shape[0]*y_test.shape[1])*100\n",
    "    \n",
    "    return error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "86b51792",
   "metadata": {},
   "outputs": [],
   "source": [
    "def rmse(y_test,y_pred):\n",
    "    error = np.sum((y_test-y_pred)**2)\n",
    "    error = error/(y_test.shape[0]*y_test.shape[1])\n",
    "    error = math.sqrt(error)\n",
    "    return error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0893c305",
   "metadata": {},
   "outputs": [],
   "source": [
    "def errorMagnitude(y_true,y_pred):\n",
    "    \n",
    "    minMag = min([min(abs(i)) for i in y_true-y_pred])\n",
    "    maxMag = max([max(abs(i)) for i in y_true-y_pred])\n",
    "    \n",
    "    return (minMag,maxMag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3011e9c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_transformation(X,C):\n",
    "    phi = np.zeros(shape=X.shape)\n",
    "    for i in range(X.shape[1]):\n",
    "        phi[:,i] = X[:,i]**C[i]\n",
    "    phi = np.prod(phi,axis=1)\n",
    "    phi = phi.reshape(phi.shape[0],1)\n",
    "    phi = np.concatenate((X,phi),axis=1)\n",
    "    return phi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "360dbe95",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,Y_train,Y_test,scaler = import_dataset(normalised=False,scaleMethod='MinMax')\n",
    "X_trainD = X_train.reshape(X_train.shape[0],X_train.shape[1],1)\n",
    "X_testD = X_test.reshape(X_test.shape[0],X_test.shape[1],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "1afb2aee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/200\n",
      "200/200 [==============================] - 1s 4ms/step - loss: 197.6749 - accuracy: 0.2618 - val_loss: 193.2772 - val_accuracy: 0.2569\n",
      "Epoch 2/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 191.7224 - accuracy: 0.2618 - val_loss: 187.8719 - val_accuracy: 0.2569\n",
      "Epoch 3/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 186.8691 - accuracy: 0.2622 - val_loss: 183.5717 - val_accuracy: 0.2573\n",
      "Epoch 4/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 183.0831 - accuracy: 0.2643 - val_loss: 180.3016 - val_accuracy: 0.2582\n",
      "Epoch 5/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 180.1138 - accuracy: 0.2634 - val_loss: 177.6430 - val_accuracy: 0.2573\n",
      "Epoch 6/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 177.7195 - accuracy: 0.2625 - val_loss: 175.4578 - val_accuracy: 0.2573\n",
      "Epoch 7/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 175.5826 - accuracy: 0.2623 - val_loss: 173.3093 - val_accuracy: 0.2569\n",
      "Epoch 8/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 173.2202 - accuracy: 0.2621 - val_loss: 170.7565 - val_accuracy: 0.2569\n",
      "Epoch 9/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 170.7973 - accuracy: 0.2620 - val_loss: 168.5524 - val_accuracy: 0.2569\n",
      "Epoch 10/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 168.7951 - accuracy: 0.2619 - val_loss: 166.7185 - val_accuracy: 0.2569\n",
      "Epoch 11/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 167.0223 - accuracy: 0.2618 - val_loss: 164.9761 - val_accuracy: 0.2569\n",
      "Epoch 12/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 165.2858 - accuracy: 0.2618 - val_loss: 163.3062 - val_accuracy: 0.2569\n",
      "Epoch 13/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 163.7652 - accuracy: 0.2618 - val_loss: 161.9186 - val_accuracy: 0.2569\n",
      "Epoch 14/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 162.4237 - accuracy: 0.2618 - val_loss: 160.6056 - val_accuracy: 0.2578\n",
      "Epoch 15/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 161.1034 - accuracy: 0.2630 - val_loss: 159.2785 - val_accuracy: 0.2596\n",
      "Epoch 16/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 159.7636 - accuracy: 0.2651 - val_loss: 157.9401 - val_accuracy: 0.2671\n",
      "Epoch 17/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 158.4310 - accuracy: 0.2722 - val_loss: 156.6220 - val_accuracy: 0.2667\n",
      "Epoch 18/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 157.1410 - accuracy: 0.2704 - val_loss: 155.3574 - val_accuracy: 0.2640\n",
      "Epoch 19/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 155.8828 - accuracy: 0.2662 - val_loss: 154.1034 - val_accuracy: 0.2596\n",
      "Epoch 20/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 154.6140 - accuracy: 0.2627 - val_loss: 152.8201 - val_accuracy: 0.2569\n",
      "Epoch 21/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 153.3245 - accuracy: 0.2618 - val_loss: 151.5287 - val_accuracy: 0.2569\n",
      "Epoch 22/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 152.0252 - accuracy: 0.2618 - val_loss: 150.2161 - val_accuracy: 0.2569\n",
      "Epoch 23/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 150.6911 - accuracy: 0.2618 - val_loss: 148.8733 - val_accuracy: 0.2569\n",
      "Epoch 24/200\n",
      "200/200 [==============================] - 1s 4ms/step - loss: 149.3882 - accuracy: 0.2618 - val_loss: 147.6321 - val_accuracy: 0.2569\n",
      "Epoch 25/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 148.1848 - accuracy: 0.2618 - val_loss: 146.4491 - val_accuracy: 0.2569\n",
      "Epoch 26/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 146.9709 - accuracy: 0.2618 - val_loss: 145.1731 - val_accuracy: 0.2569\n",
      "Epoch 27/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 145.6563 - accuracy: 0.2618 - val_loss: 143.8675 - val_accuracy: 0.2569\n",
      "Epoch 28/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 144.3780 - accuracy: 0.2618 - val_loss: 142.6170 - val_accuracy: 0.2569\n",
      "Epoch 29/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 143.1408 - accuracy: 0.2618 - val_loss: 141.3975 - val_accuracy: 0.2569\n",
      "Epoch 30/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 141.9301 - accuracy: 0.2618 - val_loss: 140.2008 - val_accuracy: 0.2569\n",
      "Epoch 31/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 140.7376 - accuracy: 0.2618 - val_loss: 139.0194 - val_accuracy: 0.2569\n",
      "Epoch 32/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 139.5605 - accuracy: 0.2618 - val_loss: 137.8515 - val_accuracy: 0.2569\n",
      "Epoch 33/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 138.3943 - accuracy: 0.2618 - val_loss: 136.6943 - val_accuracy: 0.2569\n",
      "Epoch 34/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 137.2375 - accuracy: 0.2631 - val_loss: 135.5449 - val_accuracy: 0.2671\n",
      "Epoch 35/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 136.0877 - accuracy: 0.2929 - val_loss: 134.4023 - val_accuracy: 0.2871\n",
      "Epoch 36/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 134.9440 - accuracy: 0.2881 - val_loss: 133.2652 - val_accuracy: 0.2827\n",
      "Epoch 37/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 133.8058 - accuracy: 0.2880 - val_loss: 132.1324 - val_accuracy: 0.2827\n",
      "Epoch 38/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 132.6718 - accuracy: 0.2880 - val_loss: 131.0048 - val_accuracy: 0.2827\n",
      "Epoch 39/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 131.5406 - accuracy: 0.2880 - val_loss: 129.8788 - val_accuracy: 0.2827\n",
      "Epoch 40/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 130.4109 - accuracy: 0.2880 - val_loss: 128.7532 - val_accuracy: 0.2827\n",
      "Epoch 41/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 129.2816 - accuracy: 0.2880 - val_loss: 127.6279 - val_accuracy: 0.2827\n",
      "Epoch 42/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 128.1534 - accuracy: 0.2880 - val_loss: 126.5055 - val_accuracy: 0.2827\n",
      "Epoch 43/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 127.0292 - accuracy: 0.2880 - val_loss: 125.3884 - val_accuracy: 0.2827\n",
      "Epoch 44/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 125.9074 - accuracy: 0.2880 - val_loss: 124.2718 - val_accuracy: 0.2827\n",
      "Epoch 45/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 124.7871 - accuracy: 0.2880 - val_loss: 123.1575 - val_accuracy: 0.2827\n",
      "Epoch 46/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 123.6701 - accuracy: 0.2880 - val_loss: 122.0456 - val_accuracy: 0.2827\n",
      "Epoch 47/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 122.5517 - accuracy: 0.2880 - val_loss: 120.9280 - val_accuracy: 0.2827\n",
      "Epoch 48/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 121.4271 - accuracy: 0.2880 - val_loss: 119.8080 - val_accuracy: 0.2827\n",
      "Epoch 49/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 120.3036 - accuracy: 0.2880 - val_loss: 118.6910 - val_accuracy: 0.2827\n",
      "Epoch 50/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 119.1824 - accuracy: 0.2880 - val_loss: 117.5742 - val_accuracy: 0.2827\n",
      "Epoch 51/200\n",
      "200/200 [==============================] - 0s 2ms/step - loss: 118.0424 - accuracy: 0.2880 - val_loss: 116.4132 - val_accuracy: 0.2827\n",
      "Epoch 52/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 116.8724 - accuracy: 0.2880 - val_loss: 115.2585 - val_accuracy: 0.2827\n",
      "Epoch 53/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 115.7173 - accuracy: 0.2880 - val_loss: 114.1127 - val_accuracy: 0.2827\n",
      "Epoch 54/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 114.5682 - accuracy: 0.2880 - val_loss: 112.9702 - val_accuracy: 0.2827\n",
      "Epoch 55/200\n",
      "200/200 [==============================] - 1s 3ms/step - loss: 113.4212 - accuracy: 0.2880 - val_loss: 111.8288 - val_accuracy: 0.2827\n",
      "Epoch 56/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200/200 [==============================] - 1s 3ms/step - loss: 112.2280 - accuracy: 0.2880 - val_loss: 110.5673 - val_accuracy: 0.2827\n",
      "Epoch 57/200\n",
      "178/200 [=========================>....] - ETA: 0s - loss: 110.9585 - accuracy: 0.2855 ETA: 0s - loss: 112.5008 "
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-46-c4a554179621>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     28\u001b[0m \u001b[0moptD\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mAdam\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlr\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1e-6\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     29\u001b[0m \u001b[0mcombined_model\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mopt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptD\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'mse'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'accuracy'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 30\u001b[1;33m \u001b[0mcombined_model\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mlinear_X\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_train\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mY_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m64\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m200\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mvalidation_data\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mlinear_Xt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mY_test\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\anaconda3\\envs\\keras_env\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    106\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_method_wrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    107\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_in_multi_worker_mode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 108\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    109\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    110\u001b[0m     \u001b[1;31m# Running inside `run_distribute_coordinator` already.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\keras_env\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m   1096\u001b[0m                 batch_size=batch_size):\n\u001b[0;32m   1097\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1098\u001b[1;33m               \u001b[0mtmp_logs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrain_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1099\u001b[0m               \u001b[1;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1100\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\keras_env\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    778\u001b[0m       \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    779\u001b[0m         \u001b[0mcompiler\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"nonXla\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 780\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    781\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    782\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\keras_env\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    805\u001b[0m       \u001b[1;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    806\u001b[0m       \u001b[1;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 807\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=not-callable\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    808\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    809\u001b[0m       \u001b[1;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\keras_env\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   2827\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2828\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2829\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2830\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2831\u001b[0m   \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\keras_env\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[1;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1841\u001b[0m       \u001b[0;31m`\u001b[0m\u001b[0margs\u001b[0m\u001b[0;31m`\u001b[0m \u001b[1;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1842\u001b[0m     \"\"\"\n\u001b[1;32m-> 1843\u001b[1;33m     return self._call_flat(\n\u001b[0m\u001b[0;32m   1844\u001b[0m         [t for t in nest.flatten((args, kwargs), expand_composites=True)\n\u001b[0;32m   1845\u001b[0m          if isinstance(t, (ops.Tensor,\n",
      "\u001b[1;32m~\\anaconda3\\envs\\keras_env\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1921\u001b[0m         and executing_eagerly):\n\u001b[0;32m   1922\u001b[0m       \u001b[1;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1923\u001b[1;33m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0m\u001b[0;32m   1924\u001b[0m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0;32m   1925\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n",
      "\u001b[1;32m~\\anaconda3\\envs\\keras_env\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    543\u001b[0m       \u001b[1;32mwith\u001b[0m \u001b[0m_InterpolateFunctionError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    544\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcancellation_manager\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 545\u001b[1;33m           outputs = execute.execute(\n\u001b[0m\u001b[0;32m    546\u001b[0m               \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    547\u001b[0m               \u001b[0mnum_outputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_num_outputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\keras_env\\lib\\site-packages\\tensorflow\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     57\u001b[0m   \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 59\u001b[1;33m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[0;32m     60\u001b[0m                                         inputs, attrs, num_outputs)\n\u001b[0;32m     61\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "lst = list(product([0, 1], repeat=7))\n",
    "linear_X = cross_transformation(X_train,lst[5])\n",
    "linear_Xt = cross_transformation(X_test,lst[5])\n",
    "linear_model = LinearModel()\n",
    "dnn_model = Sequential([Dense(units=20,activation='selu',kernel_initializer='he_uniform',input_dim=X_train.shape[1]),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=20,activation='selu'),\n",
    "                        Dense(units=Y_train.shape[1],activation='linear')])\n",
    "\n",
    "combined_model = WideDeepModel(linear_model, dnn_model)\n",
    "opt = Adam(lr=1e-6)\n",
    "optD = Adam(lr=1e-6)\n",
    "combined_model.compile([opt, optD], 'mse', metrics=['accuracy'])\n",
    "combined_model.fit([linear_X, X_train], Y_train, 64,epochs=200,validation_data=([linear_Xt, X_test], Y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "03f17888",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_13\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv1d_30 (Conv1D)           (None, 6, 16)             48        \n",
      "_________________________________________________________________\n",
      "max_pooling1d_21 (MaxPooling (None, 3, 16)             0         \n",
      "_________________________________________________________________\n",
      "conv1d_31 (Conv1D)           (None, 2, 32)             1056      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_22 (MaxPooling (None, 1, 32)             0         \n",
      "_________________________________________________________________\n",
      "flatten_12 (Flatten)         (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_40 (Dense)             (None, 512)               16896     \n",
      "_________________________________________________________________\n",
      "dense_41 (Dense)             (None, 512)               262656    \n",
      "_________________________________________________________________\n",
      "dense_42 (Dense)             (None, 4)                 2052      \n",
      "=================================================================\n",
      "Total params: 282,708\n",
      "Trainable params: 282,708\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "dnn_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "785d765e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 6.96075697e-03,  9.36342201e-12,  3.39866678e-01,  9.99789589e-01,\n",
       "       -1.06357835e-11, -2.05128709e-02, -2.53312297e-09])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[1,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7209bb2a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
